{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Optimization Algorithms  \n",
    "Gradient descent variations:  \n",
    "梯度下降(Gradient Descent)  \n",
    "随机梯度下降(SGD)  \n",
    "小批量梯度下降(Mini-Batch Gradient Descent)  \n",
    "动量梯度下降  \n",
    "AdaGrad: 自适应学习率方法    \n",
    "RMSProp：改进的自适应学习方法  \n",
    "Adam(Adaptive Moment Tstimation):结合了动量和自适应学习率的方法  \n",
    "Advanced optimizatin methods  \n",
    "牛顿法  \n",
    "拟牛顿法  \n",
    "共轭梯度法\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gradient Descent:\n",
      "Iteration 1: x = 8.0, loss = 64.0\n",
      "Iteration 2: x = 6.4, loss = 40.96000000000001\n",
      "Iteration 3: x = 5.12, loss = 26.2144\n",
      "Iteration 4: x = 4.096, loss = 16.777216\n",
      "Iteration 5: x = 3.2768, loss = 10.73741824\n",
      "Iteration 6: x = 2.62144, loss = 6.871947673600001\n",
      "Iteration 7: x = 2.0971520000000003, loss = 4.398046511104002\n",
      "Iteration 8: x = 1.6777216000000004, loss = 2.8147497671065613\n",
      "Iteration 9: x = 1.3421772800000003, loss = 1.801439850948199\n",
      "Iteration 10: x = 1.0737418240000003, loss = 1.1529215046068475\n",
      "Iteration 11: x = 0.8589934592000003, loss = 0.7378697629483825\n",
      "Iteration 12: x = 0.6871947673600002, loss = 0.47223664828696477\n",
      "Iteration 13: x = 0.5497558138880001, loss = 0.3022314549036574\n",
      "Iteration 14: x = 0.43980465111040007, loss = 0.19342813113834073\n",
      "Iteration 15: x = 0.35184372088832006, loss = 0.12379400392853807\n",
      "Iteration 16: x = 0.281474976710656, loss = 0.07922816251426434\n",
      "Iteration 17: x = 0.22517998136852482, loss = 0.050706024009129186\n",
      "Iteration 18: x = 0.18014398509481985, loss = 0.03245185536584268\n",
      "Iteration 19: x = 0.14411518807585588, loss = 0.020769187434139313\n",
      "Iteration 20: x = 0.11529215046068471, loss = 0.013292279957849162\n",
      "Iteration 21: x = 0.09223372036854777, loss = 0.008507059173023463\n",
      "Iteration 22: x = 0.07378697629483821, loss = 0.0054445178707350165\n",
      "Iteration 23: x = 0.05902958103587057, loss = 0.00348449143727041\n",
      "Iteration 24: x = 0.04722366482869646, loss = 0.002230074519853063\n",
      "Iteration 25: x = 0.037778931862957166, loss = 0.0014272476927059603\n",
      "Iteration 26: x = 0.030223145490365734, loss = 0.0009134385233318147\n",
      "Iteration 27: x = 0.024178516392292588, loss = 0.0005846006549323614\n",
      "Iteration 28: x = 0.01934281311383407, loss = 0.0003741444191567113\n",
      "Iteration 29: x = 0.015474250491067256, loss = 0.00023945242826029522\n",
      "Iteration 30: x = 0.012379400392853806, loss = 0.00015324955408658894\n",
      "Iteration 31: x = 0.009903520314283045, loss = 9.807971461541694e-05\n",
      "Iteration 32: x = 0.007922816251426436, loss = 6.277101735386685e-05\n",
      "Iteration 33: x = 0.006338253001141149, loss = 4.017345110647478e-05\n",
      "Iteration 34: x = 0.00507060240091292, loss = 2.5711008708143864e-05\n",
      "Iteration 35: x = 0.0040564819207303355, loss = 1.645504557321207e-05\n",
      "Iteration 36: x = 0.0032451855365842686, loss = 1.0531229166855727e-05\n",
      "Iteration 37: x = 0.002596148429267415, loss = 6.739986666787666e-06\n",
      "Iteration 38: x = 0.002076918743413932, loss = 4.313591466744106e-06\n",
      "Iteration 39: x = 0.0016615349947311456, loss = 2.7606985387162283e-06\n",
      "Iteration 40: x = 0.0013292279957849164, loss = 1.7668470647783858e-06\n",
      "Iteration 41: x = 0.001063382396627933, loss = 1.1307821214581668e-06\n",
      "Iteration 42: x = 0.0008507059173023465, loss = 7.237005577332268e-07\n",
      "Iteration 43: x = 0.0006805647338418772, loss = 4.631683569492651e-07\n",
      "Iteration 44: x = 0.0005444517870735017, loss = 2.9642774844752964e-07\n",
      "Iteration 45: x = 0.0004355614296588014, loss = 1.8971375900641897e-07\n",
      "Iteration 46: x = 0.0003484491437270411, loss = 1.2141680576410815e-07\n",
      "Iteration 47: x = 0.00027875931498163285, loss = 7.77067556890292e-08\n",
      "Iteration 48: x = 0.00022300745198530628, loss = 4.973232364097869e-08\n",
      "Iteration 49: x = 0.00017840596158824503, loss = 3.182868713022636e-08\n",
      "Iteration 50: x = 0.00014272476927059603, loss = 2.0370359763344873e-08\n",
      "Iteration 51: x = 0.00011417981541647683, loss = 1.3037030248540721e-08\n",
      "Iteration 52: x = 9.134385233318146e-05, loss = 8.34369935906606e-09\n",
      "Iteration 53: x = 7.307508186654517e-05, loss = 5.339967589802279e-09\n",
      "Iteration 54: x = 5.846006549323614e-05, loss = 3.417579257473459e-09\n",
      "Iteration 55: x = 4.676805239458891e-05, loss = 2.1872507247830133e-09\n",
      "Iteration 56: x = 3.741444191567113e-05, loss = 1.3998404638611285e-09\n",
      "Iteration 57: x = 2.9931553532536903e-05, loss = 8.958978968711224e-10\n",
      "Iteration 58: x = 2.3945242826029522e-05, loss = 5.733746539975183e-10\n",
      "Iteration 59: x = 1.9156194260823618e-05, loss = 3.669597785584117e-10\n",
      "Iteration 60: x = 1.5324955408658896e-05, loss = 2.3485425827738353e-10\n",
      "Iteration 61: x = 1.2259964326927117e-05, loss = 1.5030672529752548e-10\n",
      "Iteration 62: x = 9.807971461541694e-06, loss = 9.619630419041632e-11\n",
      "Iteration 63: x = 7.846377169233355e-06, loss = 6.156563468186643e-11\n",
      "Iteration 64: x = 6.277101735386684e-06, loss = 3.9402006196394516e-11\n",
      "Iteration 65: x = 5.021681388309347e-06, loss = 2.521728396569249e-11\n",
      "Iteration 66: x = 4.017345110647478e-06, loss = 1.6139061738043196e-11\n",
      "Iteration 67: x = 3.213876088517982e-06, loss = 1.0328999512347646e-11\n",
      "Iteration 68: x = 2.5711008708143856e-06, loss = 6.610559687902492e-12\n",
      "Iteration 69: x = 2.0568806966515085e-06, loss = 4.230758200257595e-12\n",
      "Iteration 70: x = 1.6455045573212068e-06, loss = 2.707685248164861e-12\n",
      "Iteration 71: x = 1.3164036458569655e-06, loss = 1.7329185588255109e-12\n",
      "Iteration 72: x = 1.0531229166855724e-06, loss = 1.109067877648327e-12\n",
      "Iteration 73: x = 8.424983333484579e-07, loss = 7.098034416949293e-13\n",
      "Iteration 74: x = 6.739986666787663e-07, loss = 4.542742026847547e-13\n",
      "Iteration 75: x = 5.391989333430131e-07, loss = 2.907354897182431e-13\n",
      "Iteration 76: x = 4.313591466744105e-07, loss = 1.8607071341967557e-13\n",
      "Iteration 77: x = 3.450873173395284e-07, loss = 1.1908525658859237e-13\n",
      "Iteration 78: x = 2.760698538716227e-07, loss = 7.62145642166991e-14\n",
      "Iteration 79: x = 2.2085588309729815e-07, loss = 4.8777321098687425e-14\n",
      "Iteration 80: x = 1.766847064778385e-07, loss = 3.121748550315995e-14\n",
      "Iteration 81: x = 1.4134776518227082e-07, loss = 1.9979190722022372e-14\n",
      "Iteration 82: x = 1.1307821214581666e-07, loss = 1.2786682062094318e-14\n",
      "Iteration 83: x = 9.046256971665333e-08, loss = 8.183476519740364e-15\n",
      "Iteration 84: x = 7.237005577332266e-08, loss = 5.237424972633833e-15\n",
      "Iteration 85: x = 5.789604461865813e-08, loss = 3.3519519824856534e-15\n",
      "Iteration 86: x = 4.6316835694926504e-08, loss = 2.145249268790818e-15\n",
      "Iteration 87: x = 3.7053468555941206e-08, loss = 1.3729595320261236e-15\n",
      "Iteration 88: x = 2.9642774844752963e-08, loss = 8.78694100496719e-16\n",
      "Iteration 89: x = 2.3714219875802372e-08, loss = 5.623642243179002e-16\n",
      "Iteration 90: x = 1.8971375900641896e-08, loss = 3.5991310356345614e-16\n",
      "Iteration 91: x = 1.5177100720513518e-08, loss = 2.3034438628061196e-16\n",
      "Iteration 92: x = 1.2141680576410815e-08, loss = 1.4742040721959166e-16\n",
      "Iteration 93: x = 9.713344461128652e-09, loss = 9.434906062053866e-17\n",
      "Iteration 94: x = 7.770675568902922e-09, loss = 6.038339879714475e-17\n",
      "Iteration 95: x = 6.216540455122338e-09, loss = 3.8645375230172643e-17\n",
      "Iteration 96: x = 4.9732323640978705e-09, loss = 2.4733040147310494e-17\n",
      "Iteration 97: x = 3.978585891278296e-09, loss = 1.5829145694278713e-17\n",
      "Iteration 98: x = 3.1828687130226367e-09, loss = 1.0130653244338376e-17\n",
      "Iteration 99: x = 2.5462949704181095e-09, loss = 6.483618076376561e-18\n",
      "Iteration 100: x = 2.0370359763344878e-09, loss = 4.1495155688809995e-18\n",
      "\n",
      "Stochastic Gradient Descent:\n",
      "Iteration 1: x = 8.0, loss = 64.0\n",
      "Iteration 2: x = 6.4, loss = 40.96000000000001\n",
      "Iteration 3: x = 5.12, loss = 26.2144\n",
      "Iteration 4: x = 4.096, loss = 16.777216\n",
      "Iteration 5: x = 3.2768, loss = 10.73741824\n",
      "Iteration 6: x = 2.62144, loss = 6.871947673600001\n",
      "Iteration 7: x = 2.0971520000000003, loss = 4.398046511104002\n",
      "Iteration 8: x = 1.6777216000000004, loss = 2.8147497671065613\n",
      "Iteration 9: x = 1.3421772800000003, loss = 1.801439850948199\n",
      "Iteration 10: x = 1.0737418240000003, loss = 1.1529215046068475\n",
      "Iteration 11: x = 0.8589934592000003, loss = 0.7378697629483825\n",
      "Iteration 12: x = 0.6871947673600002, loss = 0.47223664828696477\n",
      "Iteration 13: x = 0.5497558138880001, loss = 0.3022314549036574\n",
      "Iteration 14: x = 0.43980465111040007, loss = 0.19342813113834073\n",
      "Iteration 15: x = 0.35184372088832006, loss = 0.12379400392853807\n",
      "Iteration 16: x = 0.281474976710656, loss = 0.07922816251426434\n",
      "Iteration 17: x = 0.22517998136852482, loss = 0.050706024009129186\n",
      "Iteration 18: x = 0.18014398509481985, loss = 0.03245185536584268\n",
      "Iteration 19: x = 0.14411518807585588, loss = 0.020769187434139313\n",
      "Iteration 20: x = 0.11529215046068471, loss = 0.013292279957849162\n",
      "Iteration 21: x = 0.09223372036854777, loss = 0.008507059173023463\n",
      "Iteration 22: x = 0.07378697629483821, loss = 0.0054445178707350165\n",
      "Iteration 23: x = 0.05902958103587057, loss = 0.00348449143727041\n",
      "Iteration 24: x = 0.04722366482869646, loss = 0.002230074519853063\n",
      "Iteration 25: x = 0.037778931862957166, loss = 0.0014272476927059603\n",
      "Iteration 26: x = 0.030223145490365734, loss = 0.0009134385233318147\n",
      "Iteration 27: x = 0.024178516392292588, loss = 0.0005846006549323614\n",
      "Iteration 28: x = 0.01934281311383407, loss = 0.0003741444191567113\n",
      "Iteration 29: x = 0.015474250491067256, loss = 0.00023945242826029522\n",
      "Iteration 30: x = 0.012379400392853806, loss = 0.00015324955408658894\n",
      "Iteration 31: x = 0.009903520314283045, loss = 9.807971461541694e-05\n",
      "Iteration 32: x = 0.007922816251426436, loss = 6.277101735386685e-05\n",
      "Iteration 33: x = 0.006338253001141149, loss = 4.017345110647478e-05\n",
      "Iteration 34: x = 0.00507060240091292, loss = 2.5711008708143864e-05\n",
      "Iteration 35: x = 0.0040564819207303355, loss = 1.645504557321207e-05\n",
      "Iteration 36: x = 0.0032451855365842686, loss = 1.0531229166855727e-05\n",
      "Iteration 37: x = 0.002596148429267415, loss = 6.739986666787666e-06\n",
      "Iteration 38: x = 0.002076918743413932, loss = 4.313591466744106e-06\n",
      "Iteration 39: x = 0.0016615349947311456, loss = 2.7606985387162283e-06\n",
      "Iteration 40: x = 0.0013292279957849164, loss = 1.7668470647783858e-06\n",
      "Iteration 41: x = 0.001063382396627933, loss = 1.1307821214581668e-06\n",
      "Iteration 42: x = 0.0008507059173023465, loss = 7.237005577332268e-07\n",
      "Iteration 43: x = 0.0006805647338418772, loss = 4.631683569492651e-07\n",
      "Iteration 44: x = 0.0005444517870735017, loss = 2.9642774844752964e-07\n",
      "Iteration 45: x = 0.0004355614296588014, loss = 1.8971375900641897e-07\n",
      "Iteration 46: x = 0.0003484491437270411, loss = 1.2141680576410815e-07\n",
      "Iteration 47: x = 0.00027875931498163285, loss = 7.77067556890292e-08\n",
      "Iteration 48: x = 0.00022300745198530628, loss = 4.973232364097869e-08\n",
      "Iteration 49: x = 0.00017840596158824503, loss = 3.182868713022636e-08\n",
      "Iteration 50: x = 0.00014272476927059603, loss = 2.0370359763344873e-08\n",
      "Iteration 51: x = 0.00011417981541647683, loss = 1.3037030248540721e-08\n",
      "Iteration 52: x = 9.134385233318146e-05, loss = 8.34369935906606e-09\n",
      "Iteration 53: x = 7.307508186654517e-05, loss = 5.339967589802279e-09\n",
      "Iteration 54: x = 5.846006549323614e-05, loss = 3.417579257473459e-09\n",
      "Iteration 55: x = 4.676805239458891e-05, loss = 2.1872507247830133e-09\n",
      "Iteration 56: x = 3.741444191567113e-05, loss = 1.3998404638611285e-09\n",
      "Iteration 57: x = 2.9931553532536903e-05, loss = 8.958978968711224e-10\n",
      "Iteration 58: x = 2.3945242826029522e-05, loss = 5.733746539975183e-10\n",
      "Iteration 59: x = 1.9156194260823618e-05, loss = 3.669597785584117e-10\n",
      "Iteration 60: x = 1.5324955408658896e-05, loss = 2.3485425827738353e-10\n",
      "Iteration 61: x = 1.2259964326927117e-05, loss = 1.5030672529752548e-10\n",
      "Iteration 62: x = 9.807971461541694e-06, loss = 9.619630419041632e-11\n",
      "Iteration 63: x = 7.846377169233355e-06, loss = 6.156563468186643e-11\n",
      "Iteration 64: x = 6.277101735386684e-06, loss = 3.9402006196394516e-11\n",
      "Iteration 65: x = 5.021681388309347e-06, loss = 2.521728396569249e-11\n",
      "Iteration 66: x = 4.017345110647478e-06, loss = 1.6139061738043196e-11\n",
      "Iteration 67: x = 3.213876088517982e-06, loss = 1.0328999512347646e-11\n",
      "Iteration 68: x = 2.5711008708143856e-06, loss = 6.610559687902492e-12\n",
      "Iteration 69: x = 2.0568806966515085e-06, loss = 4.230758200257595e-12\n",
      "Iteration 70: x = 1.6455045573212068e-06, loss = 2.707685248164861e-12\n",
      "Iteration 71: x = 1.3164036458569655e-06, loss = 1.7329185588255109e-12\n",
      "Iteration 72: x = 1.0531229166855724e-06, loss = 1.109067877648327e-12\n",
      "Iteration 73: x = 8.424983333484579e-07, loss = 7.098034416949293e-13\n",
      "Iteration 74: x = 6.739986666787663e-07, loss = 4.542742026847547e-13\n",
      "Iteration 75: x = 5.391989333430131e-07, loss = 2.907354897182431e-13\n",
      "Iteration 76: x = 4.313591466744105e-07, loss = 1.8607071341967557e-13\n",
      "Iteration 77: x = 3.450873173395284e-07, loss = 1.1908525658859237e-13\n",
      "Iteration 78: x = 2.760698538716227e-07, loss = 7.62145642166991e-14\n",
      "Iteration 79: x = 2.2085588309729815e-07, loss = 4.8777321098687425e-14\n",
      "Iteration 80: x = 1.766847064778385e-07, loss = 3.121748550315995e-14\n",
      "Iteration 81: x = 1.4134776518227082e-07, loss = 1.9979190722022372e-14\n",
      "Iteration 82: x = 1.1307821214581666e-07, loss = 1.2786682062094318e-14\n",
      "Iteration 83: x = 9.046256971665333e-08, loss = 8.183476519740364e-15\n",
      "Iteration 84: x = 7.237005577332266e-08, loss = 5.237424972633833e-15\n",
      "Iteration 85: x = 5.789604461865813e-08, loss = 3.3519519824856534e-15\n",
      "Iteration 86: x = 4.6316835694926504e-08, loss = 2.145249268790818e-15\n",
      "Iteration 87: x = 3.7053468555941206e-08, loss = 1.3729595320261236e-15\n",
      "Iteration 88: x = 2.9642774844752963e-08, loss = 8.78694100496719e-16\n",
      "Iteration 89: x = 2.3714219875802372e-08, loss = 5.623642243179002e-16\n",
      "Iteration 90: x = 1.8971375900641896e-08, loss = 3.5991310356345614e-16\n",
      "Iteration 91: x = 1.5177100720513518e-08, loss = 2.3034438628061196e-16\n",
      "Iteration 92: x = 1.2141680576410815e-08, loss = 1.4742040721959166e-16\n",
      "Iteration 93: x = 9.713344461128652e-09, loss = 9.434906062053866e-17\n",
      "Iteration 94: x = 7.770675568902922e-09, loss = 6.038339879714475e-17\n",
      "Iteration 95: x = 6.216540455122338e-09, loss = 3.8645375230172643e-17\n",
      "Iteration 96: x = 4.9732323640978705e-09, loss = 2.4733040147310494e-17\n",
      "Iteration 97: x = 3.978585891278296e-09, loss = 1.5829145694278713e-17\n",
      "Iteration 98: x = 3.1828687130226367e-09, loss = 1.0130653244338376e-17\n",
      "Iteration 99: x = 2.5462949704181095e-09, loss = 6.483618076376561e-18\n",
      "Iteration 100: x = 2.0370359763344878e-09, loss = 4.1495155688809995e-18\n",
      "\n",
      "Momentum Gradient Descent:\n",
      "Iteration 1: x = 8.0, loss = 64.0\n",
      "Iteration 2: x = 4.6, loss = 21.159999999999997\n",
      "Iteration 3: x = 0.6199999999999992, loss = 0.384399999999999\n",
      "Iteration 4: x = -3.0860000000000007, loss = 9.523396000000005\n",
      "Iteration 5: x = -5.8042, loss = 33.68873764\n",
      "Iteration 6: x = -7.089739999999999, loss = 50.264413267599984\n",
      "Iteration 7: x = -6.828777999999999, loss = 46.63220897328399\n",
      "Iteration 8: x = -5.228156599999998, loss = 27.333621434123543\n",
      "Iteration 9: x = -2.7419660199999982, loss = 7.518377654834631\n",
      "Iteration 10: x = 0.04399870600000133, loss = 0.0019358861296745532\n",
      "Iteration 11: x = 2.5425672182000008, loss = 6.46464805906529\n",
      "Iteration 12: x = 4.28276543554, loss = 18.342079775856124\n",
      "Iteration 13: x = 4.9923907440379995, loss = 24.92396534115629\n",
      "Iteration 14: x = 4.632575372878599, loss = 21.460754585401293\n",
      "Iteration 15: x = 3.382226464259419, loss = 11.439455855536771\n",
      "Iteration 16: x = 1.580467153650273, loss = 2.4978764237673956\n",
      "Iteration 17: x = -0.3572096566280132, loss = 0.12759873878830308\n",
      "Iteration 18: x = -2.029676854552868, loss = 4.119588133907623\n",
      "Iteration 19: x = -3.128961961774664, loss = 9.790402958232752\n",
      "Iteration 20: x = -3.4925261659193474, loss = 12.197739019631296\n",
      "Iteration 21: x = -3.1212287164656933, loss = 9.74206870049008\n",
      "Iteration 22: x = -2.1628152686642657, loss = 4.67776988636728\n",
      "Iteration 23: x = -0.8676801119101276, loss = 0.7528687766043716\n",
      "Iteration 24: x = 0.4714775515506222, loss = 0.22229108161616962\n",
      "Iteration 25: x = 1.5824239383551726, loss = 2.504065520679495\n",
      "Iteration 26: x = 2.265790898808233, loss = 5.133808397122221\n",
      "Iteration 27: x = 2.4276629834543413, loss = 5.893547561234433\n",
      "Iteration 28: x = 2.08781526294497, loss = 4.358972572185976\n",
      "Iteration 29: x = 1.3643892618975424, loss = 1.8615580579813205\n",
      "Iteration 30: x = 0.4404280085753487, loss = 0.19397683073764743\n",
      "Iteration 31: x = -0.47922272112969544, loss = 0.22965441644694984\n",
      "Iteration 32: x = -1.211063833638296, loss = 1.4666756091466862\n",
      "Iteration 33: x = -1.6275080681683773, loss = 2.6487825119531636\n",
      "Iteration 34: x = -1.6768062656117753, loss = 2.8116792523949075\n",
      "Iteration 35: x = -1.3858133901884782, loss = 1.9204787524256832\n",
      "Iteration 36: x = -0.8467571242698152, loss = 0.7169976275016873\n",
      "Iteration 37: x = -0.19225506008905546, loss = 0.036962008129846326\n",
      "Iteration 38: x = 0.4352478096914394, loss = 0.18944065584119543\n",
      "Iteration 39: x = 0.9129508305555969, loss = 0.8334792190121542\n",
      "Iteration 40: x = 1.160293383222219, loss = 1.3462807351492634\n",
      "Iteration 41: x = 1.1508430039777353, loss = 1.3244396198044976\n",
      "Iteration 42: x = 0.9121690618621529, loss = 0.83205239741848\n",
      "Iteration 43: x = 0.514928701585698, loss = 0.26515156771673287\n",
      "Iteration 44: x = 0.054426637019749036, loss = 0.0029622588172795164\n",
      "Iteration 45: x = -0.37091054849355487, loss = 0.13757463498378972\n",
      "Iteration 46: x = -0.6795319057568174, loss = 0.4617636109414922\n",
      "Iteration 47: x = -0.8213847461423902, loss = 0.6746729011953988\n",
      "Iteration 48: x = -0.7847753532609277, loss = 0.615872355085814\n",
      "Iteration 49: x = -0.5948718290154259, loss = 0.35387249295615814\n",
      "Iteration 50: x = -0.3049842913913891, loss = 0.09301541799550773\n",
      "Iteration 51: x = 0.016911350748521847, loss = 0.00028599378413953045\n",
      "Iteration 52: x = 0.3032351585247373, loss = 0.09195156136552257\n",
      "Iteration 53: x = 0.5002795538183837, loss = 0.2502796319687211\n",
      "Iteration 54: x = 0.5775635988189888, loss = 0.3335797106807418\n",
      "Iteration 55: x = 0.5316065195557356, loss = 0.28260549163416265\n",
      "Iteration 56: x = 0.3839238443076606, loss = 0.1473975182279728\n",
      "Iteration 57: x = 0.17422466772286097, loss = 0.03035423484314131\n",
      "Iteration 58: x = -0.049349524748030865, loss = 0.002435375592856511\n",
      "Iteration 59: x = -0.24069639302222734, loss = 0.05793475361391053\n",
      "Iteration 60: x = -0.3647692958645587, loss = 0.13305663920552596\n",
      "Iteration 61: x = -0.40348104924974515, loss = 0.16279695710367528\n",
      "Iteration 62: x = -0.3576254174464639, loss = 0.12789593920375758\n",
      "Iteration 63: x = -0.24483026533421806, loss = 0.05994185882362362\n",
      "Iteration 64: x = -0.09434857536635316, loss = 0.008901653673660422\n",
      "Iteration 65: x = 0.059954660677995875, loss = 0.003594561337013625\n",
      "Iteration 66: x = 0.18683664098231084, loss = 0.03490793041355292\n",
      "Iteration 67: x = 0.2636630950597322, loss = 0.06951822769647736\n",
      "Iteration 68: x = 0.2800742847174649, loss = 0.0784416049599996\n",
      "Iteration 69: x = 0.2388294984659314, loss = 0.057039529337488334\n",
      "Iteration 70: x = 0.15394329114636496, loss = 0.023698536888974487\n",
      "Iteration 71: x = 0.04675704632948216, loss = 0.0021862213814573413\n",
      "Iteration 72: x = -0.05906198327160879, loss = 0.0034883178679757966\n",
      "Iteration 73: x = -0.1424867132582689, loss = 0.02030246345514414\n",
      "Iteration 74: x = -0.18907162759460922, loss = 0.035748080361274595\n",
      "Iteration 75: x = -0.19318372497839364, loss = 0.03731995159652763\n",
      "Iteration 76: x = -0.1582478676281209, loss = 0.025042387608847274\n",
      "Iteration 77: x = -0.09515602248725125, loss = 0.009054668615594265\n",
      "Iteration 78: x = -0.01934215736301831, loss = 0.0003741190514557634\n",
      "Iteration 79: x = 0.05275875272139499, loss = 0.0027834859887173034\n",
      "Iteration 80: x = 0.10709782125308796, loss = 0.01146994331715838\n",
      "Iteration 81: x = 0.13458341868099405, loss = 0.01811269658386374\n",
      "Iteration 82: x = 0.1324037726299107, loss = 0.017530759006633093\n",
      "Iteration 83: x = 0.10396133665795357, loss = 0.01080795951970836\n",
      "Iteration 84: x = 0.057570876951601416, loss = 0.003314405872976431\n",
      "Iteration 85: x = 0.004305287825564191, loss = 1.853550326095124e-05\n",
      "Iteration 86: x = -0.044494799952982146, loss = 0.0019797872228559\n",
      "Iteration 87: x = -0.07951591896307741, loss = 0.006322781368542694\n",
      "Iteration 88: x = -0.09513174227954768, loss = 0.00905004838914228\n",
      "Iteration 89: x = -0.09015963480846138, loss = 0.00812875974879512\n",
      "Iteration 90: x = -0.06765281112279144, loss = 0.004576902852816093\n",
      "Iteration 91: x = -0.03386610758113019, loss = 0.0011469132426966838\n",
      "Iteration 92: x = 0.0033151471225909696, loss = 1.0990200444423185e-05\n",
      "Iteration 93: x = 0.036115246931421825, loss = 0.0013043110609175736\n",
      "Iteration 94: x = 0.05841228737308524, loss = 0.003411995316155893\n",
      "Iteration 95: x = 0.06679716629596526, loss = 0.004461861425170837\n",
      "Iteration 96: x = 0.060984124067364226, loss = 0.003719063388263673\n",
      "Iteration 97: x = 0.04355556124815045, loss = 0.0018970869156413853\n",
      "Iteration 98: x = 0.019158742461227958, loss = 0.0003670574126956591\n",
      "Iteration 99: x = -0.006630142939247876, loss = 4.3958795394858465e-05\n",
      "Iteration 100: x = -0.02851411121182655, loss = 0.0008130545382004126\n",
      "\n",
      "Adam:\n",
      "Iteration 1: x = 9.90000000005, loss = 98.01000000098999\n",
      "Iteration 2: x = 9.800027459059471, loss = 96.04053819831964\n",
      "Iteration 3: x = 9.70010099242815, loss = 94.09195926330557\n",
      "Iteration 4: x = 9.600239395419266, loss = 92.16459644936006\n",
      "Iteration 5: x = 9.500461600614251, loss = 90.2587706247459\n",
      "Iteration 6: x = 9.40078663510384, loss = 88.37478935874698\n",
      "Iteration 7: x = 9.30123357774574, loss = 86.51294606778484\n",
      "Iteration 8: x = 9.201821516812585, loss = 84.67351922727505\n",
      "Iteration 9: x = 9.102569508342574, loss = 82.85677165420798\n",
      "Iteration 10: x = 9.003496535489624, loss = 81.06294986457367\n",
      "Iteration 11: x = 8.904621469150118, loss = 79.29228350884921\n",
      "Iteration 12: x = 8.80596303012035, loss = 77.5449848878464\n",
      "Iteration 13: x = 8.70753975301269, loss = 75.82124855029629\n",
      "Iteration 14: x = 8.60936995213032, loss = 74.12125097264443\n",
      "Iteration 15: x = 8.511471689470543, loss = 72.44515032065854\n",
      "Iteration 16: x = 8.41386274499579, loss = 70.79308629162809\n",
      "Iteration 17: x = 8.31656058928038, loss = 69.16518003517162\n",
      "Iteration 18: x = 8.219582358610113, loss = 67.56153414997459\n",
      "Iteration 19: x = 8.122944832581695, loss = 65.98223275316566\n",
      "Iteration 20: x = 8.026664414220157, loss = 64.42734161850822\n",
      "Iteration 21: x = 7.930757112605344, loss = 62.89690837914026\n",
      "Iteration 22: x = 7.835238527973439, loss = 61.39096279023938\n",
      "Iteration 23: x = 7.740123839236806, loss = 59.90951704672191\n",
      "Iteration 24: x = 7.6454277938451956, loss = 58.452566150900616\n",
      "Iteration 25: x = 7.551164699893803, loss = 57.02008832492227\n",
      "Iteration 26: x = 7.457348420368827, loss = 55.612045462777445\n",
      "Iteration 27: x = 7.363992369409019, loss = 54.228383616714254\n",
      "Iteration 28: x = 7.271109510452201, loss = 52.86903351298845\n",
      "Iteration 29: x = 7.178712356128773, loss = 51.533911092035915\n",
      "Iteration 30: x = 7.0868129697595945, loss = 50.222918068352804\n",
      "Iteration 31: x = 6.995422968313316, loss = 48.93594250560549\n",
      "Iteration 32: x = 6.9045535266777796, loss = 47.67285940275856\n",
      "Iteration 33: x = 6.814215383101605, loss = 46.43353128729855\n",
      "Iteration 34: x = 6.724418845665025, loss = 45.21780881193495\n",
      "Iteration 35: x = 6.635173799643413, loss = 44.025531351474406\n",
      "Iteration 36: x = 6.546489715632376, loss = 42.85652759688046\n",
      "Iteration 37: x = 6.458375658309729, loss = 41.71061614384762\n",
      "Iteration 38: x = 6.370840295716724, loss = 40.58760607352796\n",
      "Iteration 39: x = 6.283891908948562, loss = 39.487297523349206\n",
      "Iteration 40: x = 6.197538402152186, loss = 38.40948224615107\n",
      "Iteration 41: x = 6.111787312737562, loss = 37.353944156139825\n",
      "Iteration 42: x = 6.026645821716895, loss = 36.32045986041771\n",
      "Iteration 43: x = 5.942120764094433, loss = 35.30879917508221\n",
      "Iteration 44: x = 5.85821863923757, loss = 34.31872562511049\n",
      "Iteration 45: x = 5.77494562116777, loss = 33.349996927444806\n",
      "Iteration 46: x = 5.692307568717354, loss = 32.40236545687687\n",
      "Iteration 47: x = 5.610310035505329, loss = 31.4755786944918\n",
      "Iteration 48: x = 5.528958279692221, loss = 30.569379658577162\n",
      "Iteration 49: x = 5.44825727348018, loss = 29.683507318029683\n",
      "Iteration 50: x = 5.368211712330526, loss = 28.817696988402634\n",
      "Iteration 51: x = 5.288826023876331, loss = 27.971680710831524\n",
      "Iteration 52: x = 5.210104376512623, loss = 27.14518761415599\n",
      "Iteration 53: x = 5.132050687651301, loss = 26.33794426062219\n",
      "Iteration 54: x = 5.054668631631985, loss = 25.54967497560437\n",
      "Iteration 55: x = 4.97796164728366, loss = 24.78010216182705\n",
      "Iteration 56: x = 4.90193294513524, loss = 24.028946598602246\n",
      "Iteration 57: x = 4.826585514276093, loss = 23.29592772661982\n",
      "Iteration 58: x = 4.751922128870054, loss = 22.580763918844905\n",
      "Iteration 59: x = 4.677945354328644, loss = 21.883172738084944\n",
      "Iteration 60: x = 4.604657553151117, loss = 21.202871181791636\n",
      "Iteration 61: x = 4.532060890440492, loss = 20.53957591466027\n",
      "Iteration 62: x = 4.460157339106087, loss = 19.89300348958189\n",
      "Iteration 63: x = 4.388948684764128, loss = 19.26287055749277\n",
      "Iteration 64: x = 4.318436530348893, loss = 18.648894066651785\n",
      "Iteration 65: x = 4.24862230044748, loss = 18.05079145185964\n",
      "Iteration 66: x = 4.179507245371816, loss = 17.46828081411551\n",
      "Iteration 67: x = 4.111092444981836, loss = 16.901081091186725\n",
      "Iteration 68: x = 4.043378812273966, loss = 16.34891221954603\n",
      "Iteration 69: x = 3.9763670967491187, loss = 15.811495288109015\n",
      "Iteration 70: x = 3.910057887574375, loss = 15.288552684182584\n",
      "Iteration 71: x = 3.8444516165524085, loss = 14.779808232012428\n",
      "Iteration 72: x = 3.7795485609125072, loss = 14.284987324295804\n",
      "Iteration 73: x = 3.7153488459367767, loss = 13.803817047003738\n",
      "Iteration 74: x = 3.6518524474347864, loss = 13.33602629783544\n",
      "Iteration 75: x = 3.5890591940795495, loss = 12.881345898606945\n",
      "Iteration 76: x = 3.5269687696173144, loss = 12.439508701855873\n",
      "Iteration 77: x = 3.465580714963214, loss = 12.010249691924942\n",
      "Iteration 78: x = 3.4048944301943407, loss = 11.593306080768444\n",
      "Iteration 79: x = 3.344909176451345, loss = 11.188417398708415\n",
      "Iteration 80: x = 3.285624077759156, loss = 10.795325580350704\n",
      "Iteration 81: x = 3.2270381227769147, loss = 10.413775045855553\n",
      "Iteration 82: x = 3.1691501664867108, loss = 10.043512777742746\n",
      "Iteration 83: x = 3.111958931830195, loss = 9.684288393397727\n",
      "Iteration 84: x = 3.0554630113016397, loss = 9.335854213432484\n",
      "Iteration 85: x = 2.999660868505517, loss = 8.997965326043271\n",
      "Iteration 86: x = 2.9445508396861606, loss = 8.670379647496473\n",
      "Iteration 87: x = 2.8901311352365995, loss = 8.352857978863995\n",
      "Iteration 88: x = 2.836399841193166, loss = 8.045164059120617\n",
      "Iteration 89: x = 2.783354920722017, loss = 7.747064614707465\n",
      "Iteration 90: x = 2.7309942156032543, loss = 7.458329405658434\n",
      "Iteration 91: x = 2.6793154477178796, loss = 7.178731268379662\n",
      "Iteration 92: x = 2.6283162205424, loss = 6.908046155166287\n",
      "Iteration 93: x = 2.577994020655483, loss = 6.646053170535423\n",
      "Iteration 94: x = 2.5283462192606456, loss = 6.392534604449601\n",
      "Iteration 95: x = 2.4793700737286017, loss = 6.147275962500972\n",
      "Iteration 96: x = 2.4310627291624884, loss = 5.910065993122966\n",
      "Iteration 97: x = 2.38342121998886, loss = 5.680696711893185\n",
      "Iteration 98: x = 2.3364424715769765, loss = 5.458963422988731\n",
      "Iteration 99: x = 2.290123301888599, loss = 5.244664737853139\n",
      "Iteration 100: x = 2.2444604231601724, loss = 5.03760259113234\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "np.float64(2.2444604231601724)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "# 损失函数\n",
    "def loss_func(x):\n",
    "    return x**2\n",
    "\n",
    "# 损失函数的导数\n",
    "def gradient(x):\n",
    "    return 2*x\n",
    "\n",
    "# 梯度下降\n",
    "def gradient_descent(learning_rate, n_iterations):\n",
    "    x = 10\n",
    "    for i in range(n_iterations):\n",
    "        grad=gradient(x)\n",
    "        x -= learning_rate * grad\n",
    "        print(f\"Iteration {i+1}: x = {x}, loss = {loss_func(x)}\")\n",
    "    return x\n",
    "\n",
    "# 随机梯度下降\n",
    "def stochastic_gradient_descent(learning_rate, n_iterations):\n",
    "    x = 10\n",
    "    for i in range(n_iterations):\n",
    "        grad=gradient(x)\n",
    "        x -= learning_rate * grad\n",
    "        print(f\"Iteration {i+1}: x = {x}, loss = {loss_func(x)}\")\n",
    "    return x\n",
    "\n",
    "# 动量梯度下降\n",
    "\n",
    "def momentum_gradient_descent(learning_rate, n_iterations, momentum):\n",
    "    x = 10\n",
    "    v = 0\n",
    "    for i in range(n_iterations):\n",
    "        grad=gradient(x)\n",
    "        v = momentum * v - learning_rate * grad\n",
    "        x += v\n",
    "        print(f\"Iteration {i+1}: x = {x}, loss = {loss_func(x)}\")\n",
    "    return x\n",
    "\n",
    "\n",
    "\n",
    "# Adam 优化算法\n",
    "def adam(learning_rate, n_iterations, beta1, beta2, epsilon):\n",
    "    x = 10\n",
    "    m = 0\n",
    "    v = 0\n",
    "    for i in range(n_iterations):\n",
    "        grad=gradient(x)\n",
    "        m = beta1 * m + (1 - beta1) * grad\n",
    "        v = beta2 * v + (1 - beta2) * grad**2\n",
    "        m_hat = m / (1 - beta1**(i+1))\n",
    "        v_hat = v / (1 - beta2**(i+1))\n",
    "        x -= learning_rate * m_hat / (np.sqrt(v_hat) + epsilon)\n",
    "        print(f\"Iteration {i+1}: x = {x}, loss = {loss_func(x)}\")\n",
    "    return x\n",
    "\n",
    "\n",
    "# 参数设置\n",
    "learning_rate = 0.1\n",
    "n_iterations = 100\n",
    "print(\"Gradient Descent:\")\n",
    "gradient_descent(learning_rate, n_iterations)\n",
    "\n",
    "print(\"\\nStochastic Gradient Descent:\")\n",
    "stochastic_gradient_descent(learning_rate, n_iterations)\n",
    "\n",
    "print(\"\\nMomentum Gradient Descent:\")\n",
    "momentum_gradient_descent(learning_rate, n_iterations, 0.9)\n",
    "\n",
    "print(\"\\nAdam:\")\n",
    "adam(learning_rate, n_iterations, 0.9, 0.999, 1e-8)\n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "llm-learning",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
